#!/bin/bash

#SBATCH -A pawsey0807-gpu
#SBATCH -J quokka_benchmark
#SBATCH -o 1node_%x-%j.out
#SBATCH -t 00:10:00
#SBATCH -p gpu
#SBATCH --exclusive
#SBATCH --ntasks-per-node=8
#SBATCH --gpus-per-node=8
##SBATCH --core-spec=8
#SBATCH -N 1

# *MUST* load environment modules setup script inside the SLURM script, otherwise job will fail
. scripts/hpc_profiles/setonix-gpu.profile

# always run with GPU-aware MPI
export MPICH_GPU_SUPPORT_ENABLED=1

# use correct NIC-to-GPU binding
export MPICH_OFI_NIC_POLICY=NUMA

## run
EXE="build/src/problems/HydroBlast3D/test_hydro3d_blast"
INPUTS="inputs/benchmark_unigrid_512.in"

srun bash -c "
    case \$((SLURM_LOCALID)) in
      0) GPU=4;;
      1) GPU=5;;
      2) GPU=2;;
      3) GPU=3;;
      4) GPU=6;;
      5) GPU=7;;
      6) GPU=0;;
      7) GPU=1;;
    esac
    export ROCR_VISIBLE_DEVICES=\$((GPU));
    ${EXE} ${INPUTS}"
